package com.skytech.component.crawler.scratch.process;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.List;


public class AssertCreateHtmlPageProcessor implements PageProcessor {

    private static final Logger log = LoggerFactory.getLogger(AssertCreateHtmlPageProcessor.class);
    private String contentRegex;
    private String nextPageRegex;

    public AssertCreateHtmlPageProcessor(String contentRegex,String nextPageRegex){
        this.contentRegex = contentRegex;
        this.nextPageRegex = nextPageRegex;
    }
    @Override
    public void process(Page page) {

        if(StringUtils.isEmpty(page.getRawText())){
            log.error("页面未抓取到数据");
            return;
        }
        if(StringUtils.isEmpty(contentRegex)&&StringUtils.isEmpty(nextPageRegex)){
            log.error("无翻页，无正文页面");
            return;
        }
        if(!StringUtils.isEmpty(contentRegex)){
            List<String> contentUrl = page.getHtml().links().regex(contentRegex).all();
            if(contentUrl.isEmpty()){
                log.error("未发现正文");
            }else{
                log.info("开始存储正文内容");
                // todo
            }
        }
        if(!StringUtils.isEmpty(nextPageRegex)){
            List<String> nextPageResult = page.getHtml().links().regex(nextPageRegex).all();
            if(nextPageResult.isEmpty()){
                log.error("当前页面没有发现下一页");
            }else{
                log.info("当前页面发现下一页");
                page.addTargetRequests(nextPageResult);
            }
            return;
        }
    }

    @Override
    public Site getSite() {
        return Site.me().setRetryTimes(3).setSleepTime(1000);
    }
}
